# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem
from scrapy.pipelines.files import FilesPipeline


class XiaohuaFilter(object):
    def process_item(self, item, spider):
        if len(item['author']) > 5:
            raise DropItem
        return item


class XiaohuaPipeline(object):
    def __init__(self):
        self.file_fp = open("./data.csv", "w", encoding="utf-8")
        import csv

        self.csv_fp = csv.DictWriter(self.file_fp, ("author", "content"))
        self.csv_fp.writeheader()

    def process_item(self, item, spider):
        self.csv_fp.writerow(item)
        return item

    def __del__(self):
        self.file_fp.close()


class XiaoHuaPipeline2(object):
    def __init__(self):
        self.file_fp = open("./data.json", "w", encoding="utf-8")

    def process_item(self, item, spider):
        # import json

        self.file_fp.write(str(item) + "\n")
        return item

    def __del__(self):
        self.file_fp.close()


class StoragePipeline(object):
    def process_item(self, item, spider):
        if spider.db == "file":
            # TODO: Persist data into file
            pass
        elif spider.db == "mysql":
            # TODO: save data into myql database.
            pass
        elif spider.db == "console":
            # TODO: do others
            # print(item)
            pass
        else:
            raise DropItem

        raise DropItem


# class MyFilePipeline(FilesPipeline):
#     def file_path(self, request, response=None, info=None):
#         return "my_file.exe"

import csv


class HDFSWriterPipeline(object):
    def __init__(self):
        self.fp = open("./data.csv", "w", encoding="utf-8")
        # with open("./data.csv", "w") as fp:
        #     self.csv_writer = csv.DictWriter(f=fp, fieldnames=('author', 'content'))
        #     self.csv_writer.writeheader()

    def process_item(self, item, spider):
        # self.csv_writer.writerow(item)
        self.fp.write(str(item) + "\n")
        self.fp.flush()
        return item

    def __del__(self):
        self.fp.close()
